// File: WebPageSearcher.cpp
//
// Created by 唐建 on 2023/12/2.
//

#include "hashdup.h"
#include "Configuration.h"
#include "Logger.h"
#include "web/WebPageSearcher.h"
#include "MyRedisPlus.h"
#include "web/SplitTool.hpp"
#include "web/DistanceTool.hpp"
#include "tinyxml2.h"
#include "web/io_preamble.h"
#include <utility>
#include <string>
#include <locale>
#include <algorithm>

using namespace tinyxml2;

WebpageQuery *WebpageQuery::_pWebpageQuery = nullptr;
pthread_once_t WebpageQuery::_pOnceWebQ = PTHREAD_ONCE_INIT;

WebpageQuery *WebpageQuery::getInstance()
{
    pthread_once(&_pOnceWebQ, WebpageQueryInit);
    return _pWebpageQuery;
}

void WebpageQuery::WebpageQueryInit()
{
    _pWebpageQuery = new WebpageQuery();
    atexit(destroy);
}

void WebpageQuery::destroy()
{
    delete _pWebpageQuery;
    _pWebpageQuery = nullptr;
}

WebpageQuery::WebpageQuery()
{
    std::string dbpath = "/home/tangjian/clion/cpp_project/cpp_learn/cmake-build-debug-remote/data/weblib";
    loadLibrary(dbpath);
}

void WebpageQuery::loadLibrary(std::string &dbpath)
{
    //std::string pageLibPath = dbpath + "/pagelib.dat";
    //std::string pageOffsetPath = dbpath + "/offset.dat";
    //std::string pageInvertIndexPath = dbpath + "/invert_table.dat";

    std::string pageLibPath = Configuration::getInstance()->getConfigValue("PAGELIB_PATH");
    std::string pageOffsetPath = Configuration::getInstance()->getConfigValue("OFFSET_PATH");
    std::string pageInvertIndexPath = Configuration::getInstance()->getConfigValue("INVERT_INDEX_TABLE_PATH");
    loadOffsetLib(pageOffsetPath);
    loadPageLib(pageLibPath);
    loadIndexLib(pageInvertIndexPath);

    LogDebug("pageLib size : %uz, offsetLib size : %uz, indexLib size : ",
             _pages.size(), _offsetLib.size(), _invertIndexLib.size());
}

void WebpageQuery::loadPageLib(string &dbpath)
{
    // 加载网页库文件
    ifstream ifsPage;
    if (!open_file(ifsPage, dbpath)) {
        LogError("loadOffsetLib() ifsPage load offset error");
        return;
    }

    // 遍历偏移库map
    XMLDocument doc;
    auto itPagesBegin = _pages.begin();
    // 扩容数组
    for (const auto &item: _offsetLib) {
        // 加载偏移的网页
        int docId = item.first, openPos = item.second.first, endPos = item.second.second;
        string log;
        string webPageStr;
        int len = endPos - openPos;
        //int len = endPos;
        webPageStr.resize(len);
        ifsPage.seekg(openPos);
        ifsPage.read(&webPageStr[0], len);

        // 加载xml
        doc.Parse(webPageStr.c_str());
        //cout << webPageStr.c_str() << endl;
        XMLElement *page = doc.FirstChildElement("doc");
        if (doc.ErrorID()) {
            LogError("loadPageLib() tinyxml load webPageLib error");
            return;
        }
        // 获取xml字段内容
        string title = page->FirstChildElement("title")->GetText();
        string link = page->FirstChildElement("link")->GetText();
        const char *tmpd = page->FirstChildElement("description")->GetText();
        string content = tmpd ? tmpd : "";
        const char *tmpc = page->FirstChildElement("content")->GetText();
        if (tmpc[0] != '0') {
            content = tmpc;
        }

        // 保存到网页库
        WebPage webpage(docId, title, link, content);
        _pages.push_back(std::move(webpage));
    }
}

void WebpageQuery::loadOffsetLib(string &dbpath)
{
    // 加载网页偏移库
    ifstream ifsIndex;
    if (!open_file(ifsIndex, dbpath)) {
        LogError("loadOffsetLib() ifstream load offset error : path " + dbpath);
        return;
    }

    int docId, beginIndex, endIndex;
    string line;
    while (getline(ifsIndex, line)) {
        istringstream iss(line);
        iss >> docId >> beginIndex >> endIndex;
        _offsetLib.emplace(docId, std::make_pair(beginIndex, endIndex));
    }
    ifsIndex.close();
}

void WebpageQuery::loadIndexLib(string &dbpath)
{
    // 加载倒排索引
    ifstream ifsIndex(dbpath);
    if (!ifsIndex.good()) {
        LogError("loadIndexLib() ifstream load invertIndex error");
        return;
    }

    string word, line;
    int docid;
    double weight;
    while (getline(ifsIndex, line)) {
        istringstream iss(line);
        iss >> word;
        while (iss >> docid >> weight) {
            _invertIndexLib[word].insert(std::make_pair(docid, weight));
        }
    }
    ifsIndex.close();
}

std::string WebpageQuery::doQueryW(std::string sought, int pageSize, int pageNum)
{
    // 判空
    if (sought.empty())
        return returnNoAnswer();

    // 分词
    vector<pair<string,double>> cutWordsAndFreq;
    //vector<string> res;
    //写一个存id 和 string doule的
    Hashdup::getpHashdup()->simhasher.extract(sought,cutWordsAndFreq,strlen(sought.c_str()));

    //auto cutWords = SplitToolCppJieba::getInstance()->cut(sought);

    // 计算基准向量 Base
    // "在", "现代战争", "里", "你", "只能", "像", "牲畜", "一样", "毫无意义", "地", "死去"]
    //  0.1  0.1        0.1   0.1   0.1   0.1   0.1     0.1    0.1       0.1
    //vector<double> wordWeightVector = getQueryWordsWeightVector(cutWords);

    // 查找匹配的候选网页 和 每个单词的权重
    DocIdWordWeightVector matchVector;
    bool result = executeQuery(cutWordsAndFreq, matchVector);

    //计算basic 向量的范数(长度)
    double basicNorm = 0;
    for (const auto &item: cutWordsAndFreq) {
        basicNorm += std::pow(item.second, 2);
    }

    // 余弦排序 没有结果直接返回没有找到 todo *lambda表达式 -> 函数对象比较器可以保存basic向量
    if (result) {
        std::sort(matchVector.begin(),
                  matchVector.end(),
                  [&](DocIdWordWeightPair &lhs, DocIdWordWeightPair &rhs) -> bool {
                      // 比较余弦相似度 cosθ值越大越相似
                      // X * Y = (x1 * y1 + x2 * y2 + x3 * y3)
                      // cosθ = (X * Y) / (|X| * |Y|)
                      double lhsDotProduct = 0, rhsDotProduct = 0;
                      double lhsNorm = 0, rhsNorm = 0;
                      for (int i = 0; i < matchVector.size(); ++i) {
                          // 求左右 向量的内积
                          //lhsDotProduct += lhs.second[i] * wordWeightVector[i];
                          lhsDotProduct += lhs.second[i] * cutWordsAndFreq[i].second;
                          //rhsDotProduct += rhs.second[i] * wordWeightVector[i];
                          rhsDotProduct += rhs.second[i] * cutWordsAndFreq[i].second;
                          // 求左右 向量的范数
                          lhsNorm += std::pow(lhs.second[i], 2);
                          rhsNorm += std::pow(rhs.second[i], 2);
                      }

                      //计算左右 余弦相似度
                      double lhsCosineSimilarity = lhsDotProduct / (std::sqrt(lhsNorm) * std::sqrt(basicNorm));
                      double rhsCosineSimilarity = rhsDotProduct / (std::sqrt(lhsNorm) * std::sqrt(basicNorm));

                      return lhsCosineSimilarity > rhsCosineSimilarity;
                  }
        );
    } else {
        return returnNoAnswer();
    }

    // todo 处理分页和列表的个数 判断个数是否满足分页
    WebPageRespPVector webPagesRespPVector = getWebPagesRespPVector(matchVector, cutWordsAndFreq,
                                                                    0, 10, sought);

    // 处理json格式的字符串
    nlohmann::json msg = *webPagesRespPVector.get();
    auto sendJson = ProtocolParser::getSendmsg("200", msg);
    MyRedisPlus::getInstance()->set(sought, sendJson.dump());
    return sendJson.dump();
}

std::string WebpageQuery::returnNoAnswer()
{
    nlohmann::json msg = "没有搜索到!";
    string returnMsg = ProtocolParser::getSendmsg("200", msg);
    return returnMsg;
}

//WordWeightVector WebpageQuery::getQueryWordsWeightVector(QueryWordsVector &queryWords)
WordWeightVector WebpageQuery::getQueryWordsWeightVector(QueryWordsVector &queryWords)
{
    //词频
    std::unordered_map<string, int> wordFreq;
    for (const auto &word: queryWords) {
        ++wordFreq[word];
    }

    //计算单词权重 和 权重的平方和
    WordWeightVector weightVector;
    double wordWeightPow2Sum = 0;
    int wordOccurNum = static_cast<int>(_pages.size());
    for (auto &word: queryWords) {
        int TF = wordFreq[word];
        int DF = static_cast<int>(_invertIndexLib[word].size());
        double IDF = log2(1.0 * wordOccurNum / (DF + 1));
        double w = IDF * TF;
        wordWeightPow2Sum += pow(w, 2);
        weightVector.push_back(w);
    }

    //归一化处理
    for (auto &item: weightVector) {
        item /= sqrt(wordWeightPow2Sum);
    }

    return weightVector;
}

bool WebpageQuery::executeQuery(const CutWordsAndFreq &queryWords, DocIdWordWeightVector &matchVector)
{
    auto WaitMatchNum = queryWords.size();
    //遍历所有网页
    for (auto &page: _pages) {
        WordWeightVector wordWeight;
        //查询每一个单词 出现在文章中的权重
        for (auto &word: queryWords) {
            auto tmpSet = _invertIndexLib.find(word.first)->second;
            for (auto &item: tmpSet) {
                if (item.first == page._docId) {
                    wordWeight.push_back(item.second);
                    break;
                }
            }
        }
        // 所有单词都存在才算匹配 todo 匹配部分
        if (wordWeight.size() == WaitMatchNum) {
            matchVector.emplace_back(page._docId, wordWeight);
        }
        //matchVector.emplace_back(page._docId, wordWeight);
    }

    //返回是否找到匹配的文章
    if (matchVector.empty()) {
        return false;
    } else {
        return true;
    }
}

WebPageRespPVector WebpageQuery::getWebPagesRespPVector(DocIdWordWeightVector &matchDoc,
                                                        CutWordsAndFreq &queryWords,
                                                        int offset, int num, string &sought)
{
    WebPageRespPVector result(new vector<WebPageResp>);
    //遍历构造返回值
    auto begin = matchDoc.begin() + offset;
    int size = 0;
    for (; begin != matchDoc.end() && size <= num; ++begin, ++size) {
        //string summary = _pages[begin->first - 1].getSummaryByDP(queryWords);
        string summary = _pages[begin->first - 1].getSummaryByDP(sought);
        auto &webpage = _pages[begin->first - 1];
        result->emplace_back(webpage._docId, webpage._title, webpage._link, summary);
    }

    return result;
}

WebPageSearcher::WebPageSearcher(std::string sought, TcpConnectionPtr coon)
        : _sought(std::move(sought)), _coon(std::move(coon))
{}

//WebPageSearcher::WebPageSearcher(PRecvMsg &&recvMsg, TcpConnectionPtr coon)
//        : _recvMsg(std::move(recvMsg)), _coon(std::move(coon)), _sought(recvMsg->content)
//{
//}

std::string WebPageSearcher::doQuery()
{
    // 解析接收的内容
    LogDebug(_sought);
    auto redisCache = MyRedisPlus::getInstance()->get(_sought);
    if (!redisCache.empty()) {
        redisCache += '\n';
        _coon->sendInLoop(redisCache);
        LogDebug("redis send : " + redisCache);
        return "1";
    }

    auto sendMsg = WebpageQuery::getInstance()->doQueryW(_sought, 1, 10);
    string send = sendMsg + '\n';
    _coon->sendInLoop(send);
    LogDebug("file send : " + send);
    return "1";
}

//WebPageSearcher::WebPageSearcher(WebPageSearcher &searcher)
//{
//    this->_sought = searcher._sought;
//    this->_coon = searcher._coon;
//}

WebPage::WebPage(int id, const string &title, const string &link, const string &content)
        : _docId(id), _title(title), _link(link), _content(content)
{

}

WebPage::WebPage(WebPage &&other) noexcept
        : _docId(other._docId),
          _title(std::move(other._title)),
          _link(std::move(other._link)),
          _content(std::move(other._content))
{}

WebPage &WebPage::operator=(WebPage &&other) noexcept
{
    if (this != &other) {
        _docId = other._docId;
        _title = std::move(other._title);
        _link = std::move(other._link);
        _content = std::move(other._content);
    }
    return *this;
}

WebPageRespP WebPage::getWebPage2Resp(const QueryWordsVector &queryWords)
{
    auto *result = new WebPageResp(_docId,
                                   _title,
                                   _link,
                                   getSummary(queryWords));
    return WebPageRespP(result);
}

std::string WebPage::getSummary(const QueryWordsVector &queryWords)
{
    // 摘要列表
    vector<string> summaryVector;
    // 结束符 高亮标签
    string stopSymbol = "。", preTag = "<em>", sufTag = "</em>";
    string lineText;
    istringstream iss(_content);
    while (iss >> lineText) {
        for (const auto &word: queryWords) {
            string summary;
            //找到单词的位置
            size_t pos = lineText.find(word);
            // 如果找到单词 npos 结束提示符
            if (pos != string::npos) {
                if (DistanceTool::length(lineText) >= 100) {
                    //word向后 向前找结束符
                    size_t p1 = lineText.rfind(stopSymbol, pos);
                    size_t p2 = lineText.find(stopSymbol, pos);

                    if (p1 != string::npos && p2 != string::npos) {
                        // 如果word前后都找到结束服
                        // 头结束符 到 尾结束符 子串
                        p1 += DistanceTool::nBytesCode(lineText[p1]);
                        p2 -= DistanceTool::nBytesCode(lineText[p2]);

                        summary = lineText.substr(p1, p2 - p1);
                    } else if (p1 != string::npos) {
                        // 如果word后面没有句号
                        // 头结束符 开始
                        p1 += DistanceTool::nBytesCode(lineText[p1]);
                        // 结束符 pos向word后取10个字
                        string temp = lineText.substr(p1);
                        size_t len = DistanceTool::length(temp);
                        p2 = pos;
                        if (len > 100) {
                            for (int i = 0; i < 10; ++i) {
                                size_t bytes = DistanceTool::nBytesCode(temp[p2]);
                                p2 += (bytes - 1);
                            }
                        }
                        summary = lineText.substr(p1, p2 - p1);
                    } else if (p2 != string::npos) {
                        // word前没有结束符 计算头到关键词的字数
                        p2 -= DistanceTool::nBytesCode(lineText[p2]);
                        // 行首到结束符
                        summary = lineText.substr(0, p2);
                    }
                } else {
                    summary = lineText;
                }
                summary.append("...");
                // 拼接高亮字符
                size_t posSummaryKey = 0;
                string replaceStr = preTag.append(word).append(sufTag);
                size_t lenReplaceStr = preTag.length() + word.length() + sufTag.length();
                while ((posSummaryKey = summary.find(word, posSummaryKey)) != std::string::npos) {
                    summary.replace(posSummaryKey, word.length(), replaceStr);
                    posSummaryKey += lenReplaceStr;
                }
                summaryVector.push_back(summary);
                break;
            }
        }

        // 拿到两个摘要就停止
        if (summaryVector.size() >= 2) {
            break;
        }
    }
    string summary;
    for (const auto &item: summaryVector) {
        summary.append(item);
    }
    return summary;
}


static size_t nBytesCode(const char ch)
{
    if ((ch & 0x80) == 0) // 以0xxxxxxx开头，占用1个字节
        return 1;
     else if ((ch & 0xE0) == 0xC0) // 以110xxxxx开头，占用2个字节
         return 2;
     else if ((ch & 0xF0) == 0xE0) // 以1110xxxx开头，占用3个字节
         return 3;
     else if ((ch & 0xF8) == 0xF0) // 以11110xxx开头，占用4个字节
         return 4;
     else if ((ch & 0xFC) == 0xF8) // 以111110xx开头，占用5个字节
         return 5;
     else if ((ch & 0xFE) == 0xFC) // 以1111110x开头，占用6个字节
         return 6;
     else
         return 0; // 不是有效的UTF-8编码
 }

std::pair<int, int> findLongestCommonSubstring(const string &longStr, const string &shortStr)
{
    int len1 = longStr.size();
    int len2 = shortStr.size();

    int** dp = new int*[len1 + 1];
    for (int i = 0; i <= len1; i++) {
        dp[i] = new int[len2 + 1];
        memset(dp[i], 0, sizeof(int) * (len2 + 1));
    }

    int maxLen = 0;
    int endIndex = 0;

    for (int i = 1; i <= len1; i++) {
        for (int j = 1; j <= len2; j++) {
            if (longStr[i - 1] == shortStr[j - 1]) {
                size_t byteCount = nBytesCode(longStr[i - 1]);
                if (byteCount > 0 && i >= byteCount && j >= byteCount) {
                    bool isMatch = true;
                    for (size_t k = 1; k < byteCount; k++) {
                        if (longStr[i - k - 1] != shortStr[j - k - 1]) {
                            isMatch = false;
                            break;
                        }
                    }
                    if (isMatch) {
                        dp[i][j] = dp[i - byteCount][j - byteCount] + byteCount;
                        if (dp[i][j] > maxLen) {
                            maxLen = dp[i][j];
                            endIndex = i - 1;
                        }
                    }
                }
            }
        }
    }

    for (int i = 0; i <= len1; i++) {
        delete[] dp[i];
    }
    delete[] dp;

    // 提取最长公共子串
    //string longestCommonSubstring = longStr.substr(endIndex - maxLen, maxLen);
    //cout << longestCommonSubstring << endl;

    // 返回最长子串的起始Pos和Len
    return std::make_pair(endIndex - maxLen, maxLen);
}

std::string WebPage::getSummaryByDP(std::string searchStr)
{
    // 结束符 高亮标签
    //string stopSymbol = "。", preTag = R"(\033[1m\033[37m)", sufTag = R"(\033[0m)";
    string stopSymbol = "。", preTag = "<em>", sufTag = "</em>";
    string lineText, summary, maxLineText;
    int maxLen = 0, maxBeginPos;
    istringstream iss(_content);
    while (iss >> lineText) {
        auto poss = findLongestCommonSubstring(lineText, searchStr);
        if (poss.second > maxLen) {
            maxBeginPos = poss.first;
            maxLen = poss.second;
            maxLineText = lineText;
        }
    }

    // 拼接高亮字符
    maxLineText.insert(maxBeginPos + maxLen +(nBytesCode(maxLineText[maxBeginPos + maxLen])), sufTag);
    maxLineText.insert(maxBeginPos, preTag);

    size_t p1 = maxLineText.rfind(stopSymbol , maxBeginPos);
    size_t p2 = maxLineText.find(stopSymbol, maxBeginPos + maxLen);

    if (p1 != string::npos && p2 != string::npos) {
        // 如果word前后都找到结束服
        // 头结束符 到 尾结束符 子串
        p1 += DistanceTool::nBytesCode(maxLineText[p1]);
        p2 -= DistanceTool::nBytesCode(maxLineText[p2]);

        summary = maxLineText.substr(p1, p2 - p1);
    } else if (p1 != string::npos) {
        // 如果word后面没有句号
        // 头结束符 开始
        p1 += DistanceTool::nBytesCode(maxLineText[p1]);
        // 结束符 pos向word后取10个字
        string temp = maxLineText.substr(p1);
        size_t len = DistanceTool::length(temp);
        p2 = maxBeginPos;
        if (len > 100) {
            for (int i = 0; i < 10; ++i) {
                size_t bytes = DistanceTool::nBytesCode(temp[p2]);
                p2 += (bytes - 1);
            }
        }
        summary = maxLineText.substr(p1, p2 - p1);
    } else if (p2 != string::npos) {
        // word前没有结束符 计算头到关键词的字数
        p2 -= DistanceTool::nBytesCode(maxLineText[p2]);
        // 行首到结束符
        summary = maxLineText.substr(0, p2);
    }

    summary.append("...");
    return summary;
}

WebPageResp::WebPageResp(int docId, string &docTitle, string &url, string docSummary)
        : docId(docId), docTitle(docTitle), url(url), docSummary(std::move(docSummary))
{
}
